import tushare as ts
import datetime as dt
from .models import *
from .utils import *
from .block_profile import *
import time 
from pytdx2.hq import  TdxHq_API

INIT_KDATA_DATE = dt.date(2012,12,31)
START_KDATA_DATE = dt.date(2013,1,4)
TUSHARE_CONNECTION_MAX = 200#tushare接口访问每次访问的最大个数

pro = ts.pro_api('ee0c55d0974311764461f603097ba6f1911935595147edaba2856fa6')
tdx_api = TdxHq_API()

#初始化确定不同类数据更新的最近日期，只执行一次
def init_market_data_cur_date_oneshot_1():
    if Market_data_cur_date.objects.count() == 0:
        item = Market_data_cur_date(stock_cur_date = INIT_KDATA_DATE,market_cur_date = INIT_KDATA_DATE,block_cur_date = INIT_KDATA_DATE,
                    zt_stat_cur_date = INIT_KDATA_DATE,block_file_cur_date = INIT_KDATA_DATE,
                    stock_last_date = INIT_KDATA_DATE,market_last_date = INIT_KDATA_DATE,block_last_date = INIT_KDATA_DATE,
                    zt_stat_last_date = INIT_KDATA_DATE,block_file_last_date = INIT_KDATA_DATE)
        item.save()
        print('init_market_data_cur_date_oneshot_1')
#初始化更新各个板块（概念、指数、风格、概念）对应的个股关系。通过读取tdx文件读取。每更新一次文件，则读取一次。
def init_block_stock_file_info_2():#后续根据请求信息来更新
    #更新市场的板块基本信息，包括了行业、概念和风格等3类主要基本板块。其数据来源于tdx，根据tdx的文件读取，隔一段时间更新一次。
    date_now = dt.date.today()
    last_update_date = Market_data_cur_date.objects.values('block_file_cur_date').first()['block_file_cur_date']

    if last_update_date == INIT_KDATA_DATE:
        Block_profile.objects.all().delete()#先清除所有数据，重新读取新的文件，获取新的板块数据
        
        update_block_profile(date_now,START_KDATA_DATE,date_now)
        #文件更新，不需要是交易日
        Market_data_cur_date.objects.update(block_file_cur_date = date_now,
            block_file_last_date = START_KDATA_DATE)

def update_market_trade_date_1():
    MarketTradeDate.objects.all().delete()
    #每天18点后才更新当天的数据
    if dt.datetime.now().hour >= 18:
        date_now = dt.date.today()
    else:
        date_now = dt.date.today() - dt.timedelta(days=1)
    # date_now = dt.date.today()
    start_date_str = START_KDATA_DATE.strftime('%Y%m%d')
    end_date_str = date_now.strftime('%Y%m%d')
    df = pro.trade_cal(exchange='', start_date=start_date_str, end_date=end_date_str,is_open = 1,fields='cal_date,pretrade_date')
    tradeInfoList = df.values.tolist()
    itemList = []
    for i,tradeInfo in enumerate(tradeInfoList):
        trade_date = dt.datetime.strptime(tradeInfo[0],'%Y%m%d').date()
        pre_trade_date = dt.datetime.strptime(tradeInfo[1],'%Y%m%d').date()
        item = MarketTradeDate(seq = i,trade_date = trade_date,pre_tradedate = pre_trade_date)
        itemList.append(item)
    try:
        MarketTradeDate.objects.bulk_create(itemList)
    except Exception as e:
        print("MarketTradeDate save exception = ",e)

#更新市场上所有个股及其自2013年1月4日开始的日K线数据。
def update_stock_profile_kdata_day_2():
    date_now = dt.date.today()
    #每次更新k线数据，都是首先看其最近更新的日期
    last_update_date = Market_data_cur_date.objects.values('stock_cur_date').first()['stock_cur_date']
    if last_update_date == INIT_KDATA_DATE:
        is_init = True
    else:
        is_init = False

    if is_init:
        update_start_date = START_KDATA_DATE
    else:
        update_start_date = last_update_date + dt.timedelta(days=1)
    update_start_date = getStartTradeDate(update_start_date)
    update_end_date = getEndTradeDate()

    print('update_stock_profile_kdata_day_2:(init,start_date,end_date) = (%s,%s,%s)' % (is_init,update_start_date,update_end_date))
    if update_start_date > update_end_date:
        return
   
    #第2步：获取当前最新的股票列表#只保留那些正常上市的股票。 list_status上市状态 L上市 D退市 P暂停上市.
    df = pro.stock_basic(exchange='', list_status='L', fields='ts_code,name,area,industry,market,exchange,list_date,is_hs')#排列需要根据tushare接口的字段顺序
    stockList_All = df.values.tolist()

    #过滤掉在黑名单中的股票,第一次更新时不需要考虑黑名单。因为此时黑名单没有数据。黑名单主要来自st股以及其他自定义的黑名单。
    stock_in_BlackList = StockBlacklist.objects.values_list('code',flat=True)

    stockList_consider = []
    stockcodeList_consider = []
    stock_de_st_list = []
    #去除了st，去除了北交所，以及其他的黑名单
    for stock in stockList_All:
        stock_code = stock[0]
        stock_name = stock[1] 
        stock_list_date_str = stock[6]#获取该个股的上市日期，例如20230102
        stock_list_date = dt.datetime.strptime(stock_list_date_str,'%Y%m%d').date()#转成date格式，以方便进行计算
        #考虑在黑名单的个股，只有去除ST，才会重新加入可考虑的股票列表中
        if stock_code in stock_in_BlackList:
            #那些原先ST，现在重新解除ST的个股
            if 'ST' not in stock_name and 'BJ' not in stock_code and '退' not in stock_name:
                blackObj = StockBlacklist.objects.filter(code = stock_code)
                black_type = blackObj.values('black_type').first()['black_type']
                #如果该个股原先是因为st而在黑名单中，如果现在摘帽了，就要重新纳入考察范围内，并且从黑名单中删除。
                if black_type == 'isST':
                    stock_de_st_list.append(stock_code)
                    stockcodeList_consider.append(stock_code)
                    stockList_consider.append(stock)
                    blackObj.delete()
                    # item = Market_data_update_stat(update_date = date_now,trade_date_start = update_start_date,trade_date_end = update_end_date,
                    #         code = stock_code,name = stock_name,code_type = 'stock',event_type = 'de_st')
                    # item.save()
            else:
                continue
        else:#非黑名单中的个股，则需要重新判断是否被ST或者新增的退市股或者北交所个股
            if 'ST' in stock_name or 'st' in stock_name:
                #如果该ST个股不在黑名单列表中，则加入黑名单。适用于两种情况：1.第一次添加；2：新被ST的个股。
                item = StockBlacklist(code = stock_code,list_date = stock_list_date,black_type = 'isST')
                item.save()
                continue
            elif '退' in stock_name:
                item = StockBlacklist(code = stock_code,list_date = stock_list_date,black_type = 'isTui')
                item.save()
                continue
            elif 'BJ' in stock_code:
                item = StockBlacklist(code = stock_code,list_date = stock_list_date,black_type = 'isBJ')
                item.save()
                continue
            #注意：tuhsare的个股列表中会包含下一个交易日就要上市的个股。需要特殊对待。这些个股在更新日K线数据的时候是没有数据的。所以略去。
            elif stock_list_date > update_end_date:
                # stock_next_on_market_list.append(stock_code)
                item = Market_data_update_stat(update_date = date_now,trade_date_start = update_start_date,trade_date_end = update_end_date,
                            code = stock_code,name = stock_name,code_type = 'stock',event_type = 'next_stock')
                item.save()
                continue
            else:
                #为了不超过tushare单次数据为6000条记录，删除黑名单中的个股更新，包括st，北交所，以及其他类型不予以考虑的个股。
                stockcodeList_consider.append(stock_code)
                stockList_consider.append(stock)

    ######################################第3步：根据有效的股票列表，读取日K线数据，进行数据库表格的更新
    #获取当前所有股票列表的简介信息
    stockprofileObj = StockProfile.objects
    #如果是数据初始化，则按照单个个股进行循环，因为初始化的时间跨度比较长；否则，则按照日期进行更新，因为时间跨度短，那么遍历的时间就短。
    if is_init == True:
        #那些已经存储过数据的个股就直接不再获取数据了.代码稳定后，应该不会出现，故删除相应代码
        # stocklist_in_profile =  stockprofile.values_list('code',flat=True)
        for stock in stockList_consider:
            stock_code = stock[0]
            print('stock_code = ',stock_code)
            stock_list_date_str = stock[6]#获取该个股的上市日期，例如20230102
            stock_list_date = dt.datetime.strptime(stock_list_date_str,'%Y%m%d').date()
            #初始化时的起始日期为该个股上市日期与2013年1月4日的最大值
            update_start_date_tmp = max(stock_list_date,update_start_date)
            # if stock_code in stocklist_in_profile:
            #     last_date = stockprofile.filter(code = stock_code).values('cur_date').first()['cur_date']
            #     update_start_date = last_date + dt.timedelta(days=1)
            #     if update_start_date > update_end_date:
            #         continue

            start_date_str = update_start_date_tmp.strftime('%Y%m%d')
            #初始化时的最后日期为当前日期
            end_date_str = update_end_date.strftime('%Y%m%d')
            #按照code编码来逐一的获取数据，并保存到数据库中。last_update_date为最后的交易日期，而非当前日期，因为当前日期不一定交易。
            #这是唯一一处通过个股代码来更新K线数据的地方，其他处的K线更新统一使用根据日期来更新。采用这个的原因是初始化的时候一个个股的交易日期很多，可以批量处理。
            #而如果使用通过日期更新，那么处理计算量很大。因为每一天的个股集合都不一样。
            is_trade = update_kdata_day_by_code(stock_code,start_date_str,end_date_str)
            #只有更新成功了，才会更新stock的profile
            if is_trade:
                # if stock_code in stocklist_in_profile:
                #     stockprofile.filter(code = stock_code).update(cur_date = update_end_date)
                # else:
                item = StockProfile(code = stock_code,name = stock[1],area = stock[2],industry = stock[3],market = stock[4],
                            is_hs =stock[7], exchange_ind = stock[5],list_date = stock_list_date,cur_date = update_end_date)
                item.save()
                #第一次初始化的时候，不统计作为新股
                # item2 = Market_data_update_stat(update_date = date_now,trade_date_start = update_start_date_tmp,trade_date_end = update_end_date,
                #             code = stock_code,name = stock[1],code_type = 'stock',event_type = 'new_stock')
                # item2.save()
    #如果是数据更新，则按照日期进行循环
    else:
        stockCodeList_inDB = [stockobj['code'] for stockobj in stockprofileObj.values('code')]
        #与数据库中的列表进行对比。得到新增的股票列表（新上市的）和要删除的股票列表（退市的）。以及需要更新的时间段。
        #比较stockcodeList_toQuery和stockCodeList_inDB，得到新加入的股票列表，以及不再考虑的股票列表
        #如果stockcodeList_toQuery中有，而stockCodeList_inDB没有，说明是新上市的，或者从黑名单中解除的。
        stockcodeList_new = list(set(stockcodeList_consider).difference(set(stockCodeList_inDB)))#只用来显示那些个股是新股，其列表依然和其他已在db中的个股一样去更新数据
        #数据库中有，而新的股票列表中没有，那么就是被删除的。包括被st或者退市的。
        stockcodeList_del = list(set(stockCodeList_inDB).difference(set(stockcodeList_consider)))#需要从数据库中删除相应的日K线数据

        update_days = 0
        trade_date_list_obj = MarketTradeDate.objects.all().values('trade_date')
        trade_date_list = [ele['trade_date'] for ele in trade_date_list_obj]
        update_tmp_date = update_start_date
        while update_tmp_date <= update_end_date:
            if update_tmp_date not in trade_date_list:
                update_tmp_date += dt.timedelta(days=1)
                continue
            trade_date = update_tmp_date.strftime('%Y%m%d')
            #按照日期来更新全部被考虑的个股的日K线
            update_kdata_day_by_date(stockcodeList_consider,trade_date)
            update_days += 1
            update_tmp_date += dt.timedelta(days=1)
            
        if update_days >= 1:
            #统一更新所有个股的最新更新日期
            StockProfile.objects.update(cur_date = update_end_date)
            #增加stockcodeList_new 中个股至stock_profile中记录
            #各个个股的最新日期可能不一样，遇到停牌的就会和主板不一样
            for stock_code in stockcodeList_new:#新个股列表里，要不要区分是新去除ST，还是上市的新股。
                stock = [stock_item for stock_item in stockList_consider if stock_item[0] == stock_code][0]#新股必然在stockList_consider中
                #如果是摘帽ST的个股
                if stock_code in stock_de_st_list:
                    #本来应该取寻找摘帽的具体日期，但可以使用当前日期近似表示
                    stock_list_date = update_end_date
                    item2 = Market_data_update_stat(update_date = date_now,trade_date_start = update_start_date,trade_date_end = update_end_date,
                            code = stock_code,name = stock[1],code_type = 'stock',event_type = 'de_st')
                    item2.save()
                else:
                    #在更新间隔内新上市的股票，则取其上市日期
                    stock_list_date = dt.datetime.strptime(stock[6],'%Y%m%d').date()
                    item2 = Market_data_update_stat(update_date = date_now,trade_date_start = update_start_date,trade_date_end = update_end_date,
                            code = stock_code,name = stock[1],code_type = 'stock',event_type = 'new_stock')
                    item2.save()
                item = StockProfile(code = stock_code,name = stock[1],area = stock[2],industry = stock[3],market = stock[4],
                            is_hs =stock[7], exchange_ind = stock[5],list_date = stock_list_date,cur_date = update_end_date)
                item.save()
            #删除stockcodeList_kill中对应个股的日K线数据以及stock_profile中的记录
            for stock_code in stockcodeList_del:
                stock_name = stockprofileObj.filter(code = stock_code).values('name').first()['name']
                black_typeObj = StockBlacklist.objects.filter(code = stock_code).values('black_type').first()
                if black_typeObj:
                    black_type = black_typeObj['black_type']
                else:
                    black_type = 'isdel_qita'
                
                item2 = Market_data_update_stat(update_date = date_now,trade_date_start = update_start_date,trade_date_end = update_end_date,
                            code = stock_code,name = 'noname',code_type = 'stock',event_type = black_type)
                item2.save()
                if stock_code[:2] == '60':
                    Stock_kData_600.objects.filter(code = stock_code).delete()
                elif stock_code[:2] == '68':
                    Stock_kData_688.objects.filter(code = stock_code).delete()
                elif stock_code[:2] == '00':
                    Stock_kData_000.objects.filter(code = stock_code).delete()
                elif stock_code[:2] == '30':
                    Stock_kData_300.objects.filter(code = stock_code).delete()
                else:
                    pass
                
                stockprofileObj.filter(code = stock_code).delete()
        else:
            # html = "<html><body>当前{{date_cur}}已经是最新数据！</body></html>" 
            pass
    
    Market_data_cur_date.objects.update(stock_cur_date = update_end_date,stock_last_date = update_start_date)

    return


def update_market_profile_3():
    index_arr = ['000001.SH','399001.SZ','399006.SZ']

    last_update_date = Market_data_cur_date.objects.values('market_cur_date').first()['market_cur_date']
    if last_update_date == INIT_KDATA_DATE:
        update_start_date = START_KDATA_DATE
    else:
        update_start_date = last_update_date + dt.timedelta(days=1)
    update_start_date = getStartTradeDate(update_start_date)
    update_end_date = getEndTradeDate()
    print('update_market_profile_3. (start_date,end_date) = (%s,%s)' % (update_start_date,update_end_date))

    startdate_str = update_start_date.strftime('%Y%m%d')
    enddate_str = update_end_date.strftime('%Y%m%d')

    stockKDataObj_000 = Stock_kData_000.objects
    stockKDataObj_300 = Stock_kData_300.objects
    stockKDataObj_600 = Stock_kData_600.objects
    stockKDataObj_688 = Stock_kData_688.objects
    #如果是第一次初始化，则是按照日期开始遍历的，每一个日期都需要从数据库中读取。

    #上证指数
    szzs_data = pro.index_daily(ts_code=index_arr[0], start_date=startdate_str, end_date=enddate_str,fields=["trade_date","close","pct_chg","amount"])
    szzsInfoList = szzs_data.values.tolist()
    
    # trade_datestr_list = [item[0] for item in szzsInfoList]
    #深证成指
    szcz_data = pro.index_daily(ts_code=index_arr[1], start_date=startdate_str, end_date=enddate_str,fields=["trade_date","close","pct_chg","amount"])
    szczInfoList = szcz_data.values.tolist()
    #创业板指
    cybz_data = pro.index_daily(ts_code=index_arr[2], start_date=startdate_str, end_date=enddate_str,fields=["trade_date","close","pct_chg"])
    cybzInfoList = cybz_data.values.tolist()

    #北向资金.Tushare返回的数据最远到2014年11月17日
    bxzj_data = pro.moneyflow_hsgt(start_date=startdate_str, end_date=enddate_str,limit = 3000,fields=["trade_date","north_money"])
    bxzjInfoList = bxzj_data.values.tolist()
    # tradeDateStrList = [item[0] for item in indexInfoList]

    #数据表格是按照id排序的，第一次确实是最新日期在前，但后面的更新都是放在最后。
    for szzsInfo in szzsInfoList:
        zt_count = 0#涨停数量
        dt_count = 0#跌停数量
        inc_num = 0#上涨家数
        dec_num = 0#下跌家数
        trade_date_str = szzsInfo[0]
        print('trade_date = ',trade_date_str)
        tradedate =  dt.datetime.strptime(trade_date_str,'%Y%m%d').date()

        close_0 = round(szzsInfo[1],2)
        pct_chg_0 = round(szzsInfo[2],2)
        amount_0 = szzsInfo[3]
        szcz_item_list = [item for item in szczInfoList if item[0] == trade_date_str]#找到深圳成指当天的相关信息
        if szcz_item_list.__len__() != 0:
            szczInfo = szcz_item_list[0]
            close_1 = round(szczInfo[1],2)#获取深圳成指的收盘价
            pct_chg_1 = round(szczInfo[2],2)#获取深圳成指的涨幅
            amount_1 = szczInfo[3]#获取深圳成指的成交量
        else:
            close_1 = 0
            pct_chg_1 = 0
            amount_1 = 0
        #创业板指
        cybz_item_list = [item for item in cybzInfoList if item[0] == trade_date_str]
        if cybz_item_list.__len__() != 0:
            cybzInfo = cybz_item_list[0]
            close_2 = round(cybzInfo[1],2)
            pct_chg_2 = round(cybzInfo[2],2)
        else:
            close_2 = 0
            pct_chg_2 = 0
        #北向资金
        byzj_item_list = [item for item in bxzjInfoList if item[0] == trade_date_str]
        if byzj_item_list.__len__() != 0:
            bxzjInfo = byzj_item_list[0]
            northFlow = round(bxzjInfo[1] * 0.01,2)#单位为亿元
        else:
            northFlow = 0
        #从数据库中取值，一个个取，并统计涨跌数，统计涨停和跌停数。
        #计算涨停和跌停数量
        stock_kdata_000_day_obj = stockKDataObj_000.filter(trade_date = tradedate).values('code','close','pre_close')
        for stockobj  in stock_kdata_000_day_obj:
            stock_code = stockobj['code']
            pre_close = stockobj['pre_close']
            close = stockobj['close']
            if isZT(stock_code,pre_close,close):
                zt_count += 1
            if isDT(stock_code,pre_close,close):
                dt_count += 1
            if close >= pre_close:
                inc_num += 1
            else:
                dec_num += 1
        stock_kdata_300_day_obj = stockKDataObj_300.filter(trade_date = tradedate).values('code','close','pre_close')
        for stockobj  in stock_kdata_300_day_obj:
            stock_code = stockobj['code']
            pre_close = stockobj['pre_close']
            close = stockobj['close']
            if isZT(stock_code,pre_close,close):
                zt_count += 1
            if isDT(stock_code,pre_close,close):
                dt_count += 1
            if close >= pre_close:
                inc_num += 1
            else:
                dec_num += 1
        stock_kdata_600_day_obj = stockKDataObj_600.filter(trade_date = tradedate).values('code','close','pre_close')
        for stockobj  in stock_kdata_600_day_obj:
            stock_code = stockobj['code']
            pre_close = stockobj['pre_close']
            close = stockobj['close']
            if isZT(stock_code,pre_close,close):
                zt_count += 1
            if isDT(stock_code,pre_close,close):
                dt_count += 1
            if close >= pre_close:
                inc_num += 1
            else:
                dec_num += 1
        stock_kdata_688_day_obj = stockKDataObj_688.filter(trade_date = tradedate).values('code','close','pre_close')
        for stockobj  in stock_kdata_688_day_obj:
            stock_code = stockobj['code']
            pre_close = stockobj['pre_close']
            close = stockobj['close']
            if isZT(stock_code,pre_close,close):
                zt_count += 1
            if isDT(stock_code,pre_close,close):
                dt_count += 1
            if close >= pre_close:
                inc_num += 1
            else:
                dec_num += 1
        #由于每次查询时间比较长，所以对于每一天，每遍历一天，则存储一天的数据
        try:
            item = MarketProfile(trade_date = tradedate,index_sh = close_0,change_pct_sh = pct_chg_0,amount_sh = amount_0,
                    index_sz = close_1,amount_sz = amount_1,change_pct_sz = pct_chg_1,
                    index_cy = close_2,change_pct_cy = pct_chg_2,inc_num = inc_num,
                    dec_num = dec_num,zt_num = zt_count,dt_num = dt_count,north_money = northFlow)
            item.save()
        except Exception as e:
            print('trade_date = ',trade_date_str)
            print("MarketProfile save exception = ",e)
            
    Market_data_cur_date.objects.update(market_cur_date = update_end_date,market_last_date = update_start_date)

#依然还是计算从2013年1月4日至最新日期的日k线数据
def update_block_kdata_day_4():
    #概念和行业板块要已经有数据
    block_main_codelist = Block_profile_main.objects.all().values_list('block_code',flat=True)
    block_main_num = block_main_codelist.__len__()
    if not block_main_codelist:
        return
    block_kdata_cur_date = Market_data_cur_date.objects.values_list('block_cur_date',flat=True).first()
    #第一次更新数据
    if block_kdata_cur_date == INIT_KDATA_DATE:
        update_start_date = START_KDATA_DATE
    else:
        update_start_date = block_kdata_cur_date + dt.timedelta(days=1)#在已有基础上更新数据
    update_start_date = getStartTradeDate(update_start_date)
    update_end_date = getEndTradeDate()
    print('update_block_kdata_day_4: (start_date,end_date) = (%s,%s)' % (update_start_date,update_end_date))
    if update_start_date > update_end_date:
        return
   

    trade_date_scope = MarketTradeDate.objects.filter(trade_date__gte=update_start_date,trade_date__lte=update_end_date)
    trade_days = trade_date_scope.count()#计算交易日期数，在于tdx_api提取板块数据时候get_index_bars的入参，需要填offset

    print('trade_days = ',trade_days)
    dm = divmod(trade_days,800)
    dm0 = dm[0]#商数
    dm1 = dm[1]#余数

    with tdx_api.connect('119.147.212.81', 7709):
        for i,block_code in enumerate(block_main_codelist):
            # print('block_code = {0}({1}/{2})'.format(block_code,i + 1,block_main_num))
            data = []
            if dm0 == 0:#小于800天
                data = tdx_api.get_index_bars(9,1,block_code,0,dm1 + 1)
            elif dm0 > 0:
                #例如 从2013年1月4日至2023年6月6日共2532天，为3*800+132天。那么从2013年1月4日的后132天开始往前数
                #这一天的offset为2400。往前数132天就是2013年1月4日。
                #在从这个位置开始连续数3个800天。offset分别为1600,800,0
                data += tdx_api.get_index_bars(9,1,block_code,dm0*800,dm1 + 1)
                for i in range(dm0):
                    # print('i = ',i)
                    #参数：9表示日线，1表示上海；800位单次访问最大值
                    data += tdx_api.get_index_bars(9,1,block_code,(dm0 - 1 -i)*800,800)
            df = tdx_api.to_df(data)   
            df = df.iloc[::-1]#逆序输出

            block_kdata_list = df.values.tolist()
            itemList = []  
            for j,v_kdata in enumerate(zip(block_kdata_list,block_kdata_list[1:])):
                trade_date_str = v_kdata[0][11][:10]
                trade_date =  dt.datetime.strptime(trade_date_str,'%Y-%m-%d').date()
                open = v_kdata[0][0]
                close = v_kdata[0][1]
                pre_close = v_kdata[1][1]#唯独取了上一个交易日期的数据：板块收盘价
                high = v_kdata[0][2]
                low = v_kdata[0][3]
                vol = v_kdata[0][4]
                amount = v_kdata[0][5]
                up_num = v_kdata[0][12]
                down_num = v_kdata[0][13]
                change_pct = round((close - pre_close)*100 / pre_close,2)
                #涨停和跌停后面统一更新，因为一个个股的涨跌停可能涉及多个板块。后续集中分析涨停和跌停的数据，集中更新涨跌停相关的数据表。
                # print('block_code = %s  trade_date = %s' %(block_code,trade_date))
                ztNum = getBlockZtNumByTradeDate(block_code,trade_date)
                # print('ztNum = ',ztNum)
                     
                item = Block_Main_kData_day(code = block_code,trade_date = trade_date,open = open,high = high,low = low,
                        close = close,pre_close = pre_close,volume = vol,amount = amount,change_pct = change_pct,
                        up_num = up_num,down_num = down_num,zt_num = ztNum)
                itemList.append(item)
                # print('trade_date = {0},open = {1},close = {2},pre_close = {3}'.format(trade_date,open,close,pre_close))
            try:
                Block_Main_kData_day.objects.bulk_create(itemList)
            except Exception as e:
                print("Block_Main_kData_day save exception = ",e)
    #更新操作日期
    Market_data_cur_date.objects.update(block_cur_date = update_end_date,block_last_date = update_start_date)

#按照某个交易日取计算，得到该交易日该板块所有股票的涨停数总和
def getBlockZtNumByTradeDate(block_code:str,tradedate):
    ztNum = 0
    block_stockInfoObj = Block_profile.objects.filter(block_code = block_code).values_list('stock_code',flat=True)
    #Block_profile中的个股代码不包含.SH或者.SZ，而Tushare的数据存储的包含，所以需要添加上去。
    stockList_inBlock = []
    for stock_code in block_stockInfoObj:
        if stock_code[:2] == '00' or stock_code[:2] == '30':
            stock_code = stock_code + '.SZ'
        elif stock_code[:2] == '60' or stock_code[:2] == '68':
            stock_code = stock_code + '.SH'
        stockList_inBlock.append(stock_code)
    # print('stockList_inBlock = ',stockList_inBlock)
    ztNum = Stock_kData_zt.objects.filter(code__in = stockList_inBlock,trade_date = tradedate).count()

    return ztNum

#7统计市场中涨停股票信息。同时更新板块中的涨停数目。主要基于3。
#所有考虑的股票列表stock_profile，对应的是相应的日k线数据。那么从个股为遍历，获取各个个股从起始日期至结束日期的全部涨停数据。
#不考虑跌停，因为目前最重要的是寻找主线。而主线代表着主升，主升中最代表性的就是涨停。相对而言，这远比看跌停更有意义。

def update_market_zt_stastic_5():
    curdateObj = Market_data_cur_date.objects.values('zt_stat_cur_date').first()
    update_start_date = curdateObj['zt_stat_cur_date'] + dt.timedelta(days=1)
    update_start_date = getStartTradeDate(update_start_date)
    update_end_date = getEndTradeDate()
    print('update_market_zt_stastic_5. (start_date,end_date) = (%s,%s)' % (update_start_date,update_end_date))
    if update_start_date > update_end_date:
        return
    
     #所有交易日期列表。而不要限定起始和结束日期。
    trade_date_obj = MarketTradeDate.objects.all().values('trade_date','pre_tradedate')
    trade_date_list = [ele['trade_date'] for ele in trade_date_obj]
    trade_date_info_list = [(ele['trade_date'],ele['pre_tradedate']) for ele in trade_date_obj]

    #所有当日涨停股票的日K线基本信息.为了提高查询效率，将所有涨停数据按照分类来查询
    ztStockKDataDictList = list(Stock_kData_zt.objects.all().values('code','trade_date','close','change_pct','turn_rate_f','money_flow'))
    tmp_trade_date = update_end_date
    while tmp_trade_date >= update_start_date:
        # trade_date = tmp_date.strftime('%Y%m%d')
        if tmp_trade_date not in trade_date_list:
            tmp_trade_date  = tmp_trade_date - dt.timedelta(days=1)
            continue
        print('update_market_zt_stastic_5: trade_date = ',tmp_trade_date)
        #这里的stockcode都是当前涨停的个股，而不需要从全部的stockprofile中进行遍历，可以节省大量的遍历时间。
        stockcode_list = [ele['code'] for ele in ztStockKDataDictList if ele['trade_date'] == tmp_trade_date]
        item_list = []
        for stock_code in stockcode_list:
            ztStockKDataDictList_code = list(filter(lambda x:x['code'] == stock_code,ztStockKDataDictList))
            ztStockKDataDictList0 = list(filter(lambda x:x['trade_date'] == tmp_trade_date,ztStockKDataDictList_code))
            if ztStockKDataDictList0.__len__() == 0:
                print('error: stock_code = ',stock_code)
                continue
            else:
                ztInfo = ztStockKDataDictList0[0]
                change_pct = ztInfo['change_pct']
                close = ztInfo['close']
                turn_rate_f = ztInfo['turn_rate_f']
                money_flow = ztInfo['money_flow']
                ztTimes = 1
                previous_trade_date = [tup[1] for tup in trade_date_info_list if tup[0] == tmp_trade_date][0]
                while previous_trade_date >= START_KDATA_DATE:
                    ztStockKDataDictList0_pre = list(filter(lambda x:x['trade_date'] == previous_trade_date,ztStockKDataDictList_code))
                    if ztStockKDataDictList0_pre.__len__() == 0:#如果没有连续涨停，则退出累计
                        break
                    else:
                        ztTimes += 1
                        previous_trade_date = [tup[1] for tup in trade_date_info_list if tup[0] == previous_trade_date][0]
            item = MarketZtStock(trade_date = tmp_trade_date,code = stock_code,zt_ratio = change_pct,close = close,
                    turn_rate_f = turn_rate_f,money_flow = money_flow,zt_times = ztTimes)
            item_list.append(item)
        try:
            MarketZtStock.objects.bulk_create(item_list)
        except Exception as e:
            print("MarketZtStock save exception = ",e)

        tmp_trade_date  = tmp_trade_date - dt.timedelta(days=1)
    
    Market_data_cur_date.objects.update(zt_stat_cur_date = update_end_date,zt_stat_last_date = update_start_date)     
    return

#为了保证更新数据时的实时性，尽量在一次访问api接口中带回更多的数据。当前个股一次约为4800条数据，离tushare的6000条的限制还有一定距离。
#每一天访问3次接口，大概1s左右。那么即使是间隔30天，也不到30s就可以更新完毕数据。
def update_kdata_day_by_date(stockcodeList,trade_date_str):

    #1.基本数据，8个数据项
    data_daily = pro.query('daily', trade_date=trade_date_str,fields=["ts_code","trade_date","open","high","low","close",
                                                                                        "pre_close","pct_chg","vol","amount"])
    result_list_0 = data_daily.values.tolist()

    #过滤，只取考虑的个股，即stockcodeList中的。
    result_list_0 = [rs for rs in result_list_0 if rs[0] in stockcodeList]
    stockcodelist_normal = [rs[0] for rs in result_list_0 if rs[0] in stockcodeList]#去除那些正处于停牌的个股
    # print('result_list_0 = ',result_list_0)
     #2.每日指标,6个数据项
    data_daily_basic = pro.query('daily_basic', trade_date=trade_date_str, fields=["ts_code","trade_date","turnover_rate","turnover_rate_f",
                                                                                        "volume_ratio","pe","pb","total_mv"])
    result_list_1 = data_daily_basic.values.tolist()
    result_list_1 = [rs for rs in result_list_1 if rs[0] in stockcodeList]
    #3.资金流向,1个数据项
    data_daily_moneyflow = pro.query('moneyflow', trade_date=trade_date_str,fields=["ts_code","trade_date","net_mf_amount"])
    result_list_2 = data_daily_moneyflow.values.tolist()
    result_list_2 = [rs for rs in result_list_2 if rs[0] in stockcodeList]
    querysetList_zt = []
    for stock_code in stockcodelist_normal:
        # print('stock_code = ',stock_code)
        querysetList = []
        
        #如果该个股在交易日正处于停牌状态，那么这一天就没有数据
        rs0_item_list = [rs for rs in result_list_0 if rs[0] == stock_code]
        if rs0_item_list.__len__() != 0:
            rs0_item = rs0_item_list[0]
        else:
            continue
            
        rs1_item_list = [rs for rs in result_list_1 if rs[0] == stock_code]
        if rs1_item_list.__len__() != 0:
            rs1_item = rs1_item_list[0]
            rs0_item.extend(rs1_item[2:])
        else:
            rs0_item.extend([0,0,0,0,0,0])
            
        rs2_item_list = [rs for rs in result_list_2 if rs[0] == stock_code]
        if rs2_item_list.__len__() != 0:
            rs2_item = rs2_item_list[0]
            rs0_item.extend(rs2_item[2:])
        else:
            rs0_item.extend([0])
        rslt = rs0_item

        rs_code = rslt[0]
        rs_trade_date_str = rslt[1]
        rs_trade_date = dt.datetime.strptime(rs_trade_date_str,'%Y%m%d').date()
        rs_open = rslt[2]
        rs_high = rslt[3]
        rs_low = rslt[4]
        rs_close = rslt[5]
        rs_pre_close = rslt[6]
        is_Zt = isZT(rs_code,rs_pre_close,rs_close)
        rs_pct_chg = rslt[7]
        rs_vol = rslt[8]
        rs_amount = rslt[9]
        rs_turnover_rate = rslt[10]
        rs_turnover_rate_f = rslt[11]
        rs_volume_ratio = rslt[12]
        rs_pe= rslt[13]
        rs_pb = rslt[14]
        rs_total_mv = rslt[15]
        rs_net_mf_amount = rslt[16]
        
        item = Stock_kData_600(code=rs_code,trade_date=rs_trade_date,open=rs_open,high=rs_high,low=rs_low,close=rs_close,pre_close=rs_pre_close,
            is_zt = is_Zt,change_pct=rs_pct_chg,volume=rs_vol,amount=rs_amount,turn_rate=rs_turnover_rate,turn_rate_f=rs_turnover_rate_f,
            volume_ratio=rs_volume_ratio,pe=rs_pe,pb=rs_pb,total_mv=rs_total_mv,money_flow=rs_net_mf_amount)
        querysetList.append(item)
        if is_Zt == True:
            querysetList_zt.append(item)
        #实际上，这里一次只存储一条数据,但为了方便存储在不同的数据表中，所以使用了批量存储。
        try:
            if stock_code[:2] == '60':
                Stock_kData_600.objects.bulk_create(querysetList)
            elif stock_code[:2] == '68':
                Stock_kData_688.objects.bulk_create(querysetList)
            elif stock_code[:2] == '00':
                Stock_kData_000.objects.bulk_create(querysetList)
            elif stock_code[:2] == '30':
                Stock_kData_300.objects.bulk_create(querysetList)
            else:
                pass
        except Exception as e:
            print('stock_code = ',stock_code)
            print("Stock_kData save exception = ",e)
    try:
         Stock_kData_zt.objects.bulk_create(querysetList_zt)
    except Exception as e:
        print("Stock_kData_zt save exception = ",e)
    return

#更新一个个股的日K线数据。如果该个股在数据中已有数据，那么当这次更新时该个股st了，则删除已有的K线数据，并将该个股加入至黑名单中。
#如果不在数据库中，而该个股不再是st了，则从不是st的这一天开始存日K线数据。并将该个股从黑名单中删除。

#每个个股在数据表中的起始日期有3种：上市日，2013年1月4日，最后一次去除ST即摘帽后的第一个交易日；
#结束日期就是当前更新的当天日期。但是最后一次摘帽的日期需要先获取全部数据后才可以获得。
#所以这里的入参的起始日期其实只有两种：上市日、2013年1月4日的的最大值（针对首次获取数据），以及上一次记录K线数据的日期的第二天。
#这里的数据都是非ST的，st在tushare数据获取信息时直接通过名称来过滤掉。
def update_kdata_day_by_code(stock_code:str,start_date_str:str,end_date_str:str):
    time_start = time.time()
    #获取单个个股的3类数据，最终将它们拼在一起，存入数据库
    #1.基本数据，8个数据项
    data_daily = pro.query('daily', ts_code=stock_code, start_date=start_date_str, end_date=end_date_str,fields=["ts_code","trade_date","open","high","low","close",
                                                                                        "pre_close","pct_chg","vol","amount"])
    result_list_0 = data_daily.values.tolist()
    if result_list_0.__len__() == 0:
        return False
   
    #2.每日指标,6个数据项
    data_daily_basic = pro.query('daily_basic', ts_code=stock_code, start_date=start_date_str, end_date=end_date_str,fields=["ts_code","trade_date","turnover_rate","turnover_rate_f",
                                                                                        "volume_ratio","pe","pb","total_mv"])
    result_list_1 = data_daily_basic.values.tolist()
    #3.资金流向,1个数据项
    data_daily_moneyflow = pro.query('moneyflow', ts_code=stock_code, start_date=start_date_str, end_date=end_date_str,fields=["ts_code","trade_date","net_mf_amount"])
    result_list_2 = data_daily_moneyflow.values.tolist()

    #基于Tushare返回数据的正确性，直接扩展基本数据
    querysetList = []
    querysetList_zt = []
    for rs0_item in result_list_0:
        trade_date = rs0_item[1]
        rs1_item_list = [rs1 for rs1 in result_list_1 if rs1[1] == trade_date]
        if rs1_item_list.__len__() !=0:
            rs1_item = rs1_item_list[0]
            rs0_item.extend(rs1_item[2:])
        else:
            rs0_item.extend([0,0,0,0,0,0])
        
        rs2_item_list = [rs2 for rs2 in result_list_2 if rs2[1] == trade_date]
        if rs2_item_list.__len__() !=0:
            rs2_item = rs2_item_list[0]
            rs0_item.extend(rs2_item[2:])
        else:
            rs0_item.extend([0])
        rslt = rs0_item
        # print('rslt = ',rslt)
        rs_code = rslt[0]
        rs_trade_date_str = rslt[1]
        rs_trade_date = dt.datetime.strptime(rs_trade_date_str,'%Y%m%d').date()
        rs_open = rslt[2]
        rs_high = rslt[3]
        rs_low = rslt[4]
        rs_close = rslt[5]
        rs_pre_close = rslt[6]
        is_Zt = isZT(rs_code,rs_pre_close,rs_close) 

        rs_pct_chg = rslt[7]
        rs_vol = rslt[8]
        rs_amount = rslt[9]
        rs_turnover_rate = rslt[10]
        rs_turnover_rate_f = rslt[11]
        rs_volume_ratio = rslt[12]
        rs_pe= rslt[13]
        #pe值经常为空，通过model中null或者blank使其可以为null或者为空，就不用判断是否为nan。有的时候压根就没有值。
        # if np.isnan(rs_pe):
        #     rs_pe = 0.0
        rs_pb = rslt[14]
        rs_total_mv = rslt[15]
        rs_net_mf_amount = rslt[16]
        
        item = Stock_kData_600(code=rs_code,trade_date=rs_trade_date,open=rs_open,high=rs_high,low=rs_low,close=rs_close,pre_close=rs_pre_close,
                is_zt = is_Zt,change_pct=rs_pct_chg,volume=rs_vol,amount=rs_amount,turn_rate=rs_turnover_rate,turn_rate_f=rs_turnover_rate_f,
                volume_ratio=rs_volume_ratio,pe=rs_pe,pb=rs_pb,total_mv=rs_total_mv,money_flow=rs_net_mf_amount)
        querysetList.append(item)
        if is_Zt == True:
            querysetList_zt.append(item)
    try:
        #由于这四个数据表格的格式定义完全一样，可以根据不同code存入不同的数据表格
        if stock_code[:2] == '60':
            Stock_kData_600.objects.bulk_create(querysetList)
        elif stock_code[:2] == '68':
            Stock_kData_688.objects.bulk_create(querysetList)
        elif stock_code[:2] == '00':
            Stock_kData_000.objects.bulk_create(querysetList)
        elif stock_code[:2] == '30':
            Stock_kData_300.objects.bulk_create(querysetList)
        else:
            pass
        #涨停的股票单独记录，提高后续搜索效率
        Stock_kData_zt.objects.bulk_create(querysetList_zt)
    except Exception as e:
        print('queryList = ',querysetList)
        print("Stock_kData bulk_create exception = ",e)

    time_end = time.time()
    time_sum = time_end - time_start  # 计算的时间差为程序的执行时间，单位为秒/s
    # print('time_sum = ',time_sum)
    #由于每分钟200次tushare接口访问，导致每个个股的循环不能短于0.9s
    time_len_limit =  (60 * 3) /TUSHARE_CONNECTION_MAX
    if time_sum < time_len_limit:
        time_offset = time_len_limit - time_sum + 0.01
        time.sleep(time_offset)
    return True


#为了保证更新数据时的实时性，尽量在一次访问api接口中带回更多的数据。当前个股一次约为4800条数据，离tushare的6000条的限制还有一定距离。
#每一天访问3次接口，大概1s左右。那么即使是间隔30天，也不到30s就可以更新完毕数据。


#更新一个个股的日K线数据。如果该个股在数据中已有数据，那么当这次更新时该个股st了，则删除已有的K线数据，并将该个股加入至黑名单中。
#如果不在数据库中，而该个股不再是st了，则从不是st的这一天开始存日K线数据。并将该个股从黑名单中删除。

def update_block_profile(date_now,update_start_date,update_end_date):
    #概念板块更新
    updateGnBlockInfo(date_now,update_start_date,update_end_date)
    #行业板块更新
    updateHyBlockInfo(date_now,update_start_date,update_end_date)
    #指数板块更新
    updateZsBlockInfo()
    #风格板块更新
    updateFgBlockInfo()

#获取更新日期前最新一个交易日

def getStartTradeDate(update_start_date):
    date_now = dt.date.today()
    if update_start_date >= date_now:
        return update_start_date
    tradedateObj = MarketTradeDate.objects.all().values_list('trade_date',flat=True)
    tradedateList = [ele for ele in tradedateObj]

    while update_start_date not in tradedateList:
        update_start_date += dt.timedelta(days=1)
        if update_start_date >= date_now:
            break 
    
    return update_start_date
        

def getEndTradeDate():
    #每天18点后才更新当天的数据
    if dt.datetime.now().hour >= 18:
        date_now = dt.date.today()
    else:
        date_now = dt.date.today() - dt.timedelta(days=1)
    
    temp_date = date_now
    tradedateObj = MarketTradeDate.objects.all().values_list('trade_date',flat=True)
    tradedateList = [ele for ele in tradedateObj]
    if tradedateList.__len__() == 0:
        return START_KDATA_DATE
    while temp_date >= START_KDATA_DATE:
        if temp_date in tradedateList:
            return temp_date
        else:
            temp_date = temp_date - dt.timedelta(days=1)

#计算两个日期之间有多少个交易日（含两端的交易日）
def getTradeDateInterval(start_date:str,end_date:str):
    # print('start_date =',start_date)
    # print('end_date =',end_date)
    df = pro.trade_cal(exchange='', start_date=start_date, end_date=end_date,fields='cal_date,is_open')
    df = df[df['is_open'] == 1]
    # print('df =',df)
    return df['is_open'].count()



def update_block_stock_file():#后续根据请求信息来更新
    #更新市场的板块基本信息，包括了行业、概念和风格等3类主要基本板块。其数据来源于tdx，根据tdx的文件读取，隔一段时间更新一次。

    date_now = dt.date.today()
    last_update_date = Market_data_cur_date.objects.values('block_file_cur_date').first()['block_file_cur_date']
    print('update_block_stock_file_info_5.last_update_date = ',last_update_date)

    Block_profile.objects.all().delete()#先清除所有数据，重新读取新的文件，获取新的板块数据
    
    update_block_profile(date_now,last_update_date,date_now)

    Market_data_cur_date.objects.update(block_file_cur_date = date_now,block_file_last_date = last_update_date)